package crawler;

import java.net.*;
import org.htmlparser.*;
import org.htmlparser.filters.*;
import org.htmlparser.nodes.*;
import org.htmlparser.tags.*;
import org.htmlparser.util.*;
import java.util.*;

public class webParser {
	
	private webGetter wbGet;
	private URLConnection conn; 
	private Parser parser;
	private NodeList list;
	
	public webParser(String url) throws Exception{
		wbGet = new webGetter(url);
		conn = wbGet.getURLConnection();
		if(wbGet.getResponseCode() < 400)
			parser = new Parser(conn);
		else
			parser = null;
	}
	
	public LinkedList<String> getLinksTagsOnPage() throws Exception{
		if(parser!=null){
			LinkedList<String> urls = new LinkedList<String>();
			this.list = this.parser.parse(new LinkRegexFilter("\\w"));
			for (NodeIterator itr = list.elements(); itr.hasMoreNodes();){
			    Node n = itr.nextNode(); 
				processNodes(n, false);
				LinkTag lTag = (LinkTag)n;
				String link = lTag.extractLink();
				urls.add(link);
			}
			return urls;
		}
		else
			return null;
	}
	
	
	public void visitNodes() throws Exception{
		 for (NodeIterator itr = parser.elements(); itr.hasMoreNodes();)
		     processNodes (itr.nextNode(), true);
		 	//assuming that someone who wants to visit the nodes also wants to know their contents
	}
	
	private void processNodes(Node node, boolean verbose) throws Exception{
		if (node instanceof TextNode)
	     {
	         // downcast to TextNode
	         TextNode text = (TextNode)node;
	         // do whatever processing you want with the text
	         if(verbose)
	         System.out.println (text.getText());
	     }
	     if (node instanceof RemarkNode)
	     {
	         // downcast to RemarkNode
	         RemarkNode remark = (RemarkNode)node;
	         if(verbose)
	         System.out.println(remark.getText());
	         // do whatever processing you want with the comment
	     }
	     else if (node instanceof TagNode)
	     {
	         // downcast to TagNode
	         TagNode tag = (TagNode)node;
	         // do whatever processing you want with the tag itself
	         // ...
	         // process recursively (nodes within nodes) via getChildren()
	         NodeList nl = tag.getChildren ();
	         if(verbose)
	         System.out.print(tag.getTagName()+":");
	         if (null != nl)
	             for (NodeIterator i = nl.elements (); i.hasMoreNodes(); )
	                 processNodes (i.nextNode(), verbose);
	     }
	}
	
}
